# import data
# creating a pdf file object
pdfFileObj = open('The_Shadow_Over_Innsmouth.pdf', 'rb')

# creating a pdf reader object
pdfReader = PyPDF2.PdfReader(pdfFileObj)
# how many pages
print(len(pdfReader.pages))
# creating a page object
pageObj = pdfReader.pages

text = []
for page in pageObj:
  text.append(page.extract_text())

# closing the pdf file object
pdfFileObj.close()
# text
# we can see that it added the page num in the begiing of each page. 
# we will remove them in next step:

type(book)

DF = pd.DataFrame(
    nltk.sent_tokenize(book),
    columns = ["sentences"]
)

DF.head(10)

# for IE, we want sentence and/or paragraph level structure 

DF_POS.head(20)

DF_POS['upos'].value_counts()
# punct is not pos. so most common Nouns and verbs
#  x are thins it could not figure it out

DF_POS2 = pd.crosstab(DF_POS['token'], DF_POS['upos'])

# convert to true false to add up how many times not zero 
DF_POS2['total'] = DF_POS2.astype(bool).sum(axis=1)
#print out the rows that aren't 1 
DF_POS2[DF_POS2['total'] > 1]

# textacy KPE
# build an english language for textacy pipe
en = textacy.load_spacy_lang("en_core_web_sm", disable=("parser"))

# build a processor for textacy using spacy and process text
doc = textacy.make_spacy_doc(book, lang = en)

# text rank algorithm  
print([kps for kps, weights in textacy.extract.keyterms.textrank(doc, normalize = "lemma",  topn = 5)])

terms = set([term for term, weight in textacy.extract.keyterms.textrank(doc)])
print(textacy.extract.utils.aggregate_term_variants(terms))

TR_keywords = keywords.keywords(book, scores = True)
print(TR_keywords[0:10])

print(TR_keywords)

print(DF_NER['entity'].value_counts())

DF_NER2 = pd.crosstab(DF_NER['token'], DF_NER['entity'])
print(DF_NER2)

# convert to true false to add up how many times not zero 
DF_NER2['total'] = DF_NER2.astype(bool).sum(axis=1)
#print out the rows that aren't 1 
DF_NER2[DF_NER2['total'] > 1]

DF_dev.head(10)

L_dev

DF_combined = pd.concat([DF_dev, pd.DataFrame(L_dev, columns = ["location1", "location2", "family1", "family2"])], axis = 1)
DF_combined.head()

DF_combined['location_yes'] = DF_combined['location1'] + DF_combined["location2"]
DF_combined['family_yes'] = DF_combined['family1'] + DF_combined["family2"]

print(DF_combined['location_yes'].value_counts())
print(DF_combined['family_yes'].value_counts())

# locations only
DF_loc = DF_combined[DF_combined['location_yes'] > 0]
DF_loc = DF_loc[['person1', 'person2']].reset_index(drop = True)

cooc_loc = DF_loc.groupby(by=["person1", "person2"], as_index=False).size()

# family only
DF_fam = DF_combined[DF_combined['family_yes'] > 0]
DF_fam = DF_fam[['person1', 'person2']].reset_index(drop = True)

cooc_fam = DF_fam.groupby(by=["person1", "person2"], as_index=False).size()

# take out issues where entity 1 == entity 2
cooc_loc = cooc_loc[cooc_loc['person1'] != cooc_loc['person2']]
cooc_fam = cooc_fam[cooc_fam['person1'] != cooc_fam['person2']]

print(cooc_loc.head())
print(cooc_fam.head())

# start by plotting the whole thing for location 
cooc_loc_small = cooc_loc[cooc_loc['size']>1]
graph = nx.from_pandas_edgelist(
           cooc_loc_small[['person1', 'person2', 'size']] \
           .rename(columns={'size': 'weight'}),
           source='person1', target='person2', edge_attr=True)

pos = nx.kamada_kawai_layout(graph, weight='weight')

_ = plt.figure(figsize=(20, 20))
nx.draw(graph, pos, 
        node_size=1000, 
        node_color='skyblue',
        alpha=0.8,
        with_labels = True)
plt.title('Graph Visualization', size=15)

for (node1,node2,data) in graph.edges(data=True):
    width = data['weight'] 
    _ = nx.draw_networkx_edges(graph,pos,
                               edgelist=[(node1, node2)],
                               width=width,
                               edge_color='#505050',
                               alpha=0.5)

plt.show()
plt.close()

cooc_fam
graph = nx.from_pandas_edgelist(
           cooc_fam[['person1', 'person2', 'size']] \
           .rename(columns={'size': 'weight'}),
           source='person1', target='person2', edge_attr=True)

pos = nx.kamada_kawai_layout(graph, weight='weight')

_ = plt.figure(figsize=(20, 20))
nx.draw(graph, pos, 
        node_size=1000, 
        node_color='skyblue',
        alpha=0.8,
        with_labels = True)
plt.title('Graph Visualization', size=15)

for (node1,node2,data) in graph.edges(data=True):
    width = data['weight'] 
    _ = nx.draw_networkx_edges(graph,pos,
                               edgelist=[(node1, node2)],
                               width=width,
                               edge_color='#505050',
                               alpha=0.5)

plt.show()
plt.close()